{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install plotly kaleido datasets nbformat -U -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import datasets\n",
    "import pandas as pd\n",
    "from dotenv import load_dotenv\n",
    "from huggingface_hub import login\n",
    "\n",
    "\n",
    "load_dotenv(override=True)\n",
    "login(os.getenv(\"HF_TOKEN\"))\n",
    "\n",
    "pd.set_option(\"max_colwidth\", None)\n",
    "\n",
    "OUTPUT_DIR = \"output\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_ds = datasets.load_dataset(\"gaia-benchmark/GAIA\", \"2023_all\")[\"validation\"]\n",
    "eval_ds = eval_ds.rename_columns({\"Question\": \"question\", \"Final answer\": \"true_answer\", \"Level\": \"task\"})\n",
    "eval_df = pd.DataFrame(eval_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load all results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "\n",
    "results = []\n",
    "for f in glob.glob(f\"{OUTPUT_DIR}/validation/*.jsonl\"):\n",
    "    df = pd.read_json(f, lines=True)\n",
    "    df[\"agent_name\"] = f.split(\"/\")[-1].split(\".\")[0]\n",
    "    results.append(df)\n",
    "\n",
    "result_df = pd.concat(results)\n",
    "result_df[\"prediction\"] = result_df[\"prediction\"].fillna(\"No prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "from scripts.gaia_scorer import check_close_call, question_scorer\n",
    "\n",
    "\n",
    "result_df[\"is_correct\"] = result_df.apply(lambda x: question_scorer(x[\"prediction\"], x[\"true_answer\"]), axis=1)\n",
    "result_df[\"is_near_correct\"] = result_df.apply(\n",
    "    lambda x: check_close_call(x[\"prediction\"], x[\"true_answer\"], x[\"is_correct\"]),\n",
    "    axis=1,\n",
    ")\n",
    "\n",
    "result_df[\"count_steps\"] = result_df[\"intermediate_steps\"].apply(len)\n",
    "\n",
    "\n",
    "def find_attachment(question):\n",
    "    matches = eval_df.loc[eval_df[\"question\"].apply(lambda x: x in question), \"file_name\"]\n",
    "\n",
    "    if len(matches) == 0:\n",
    "        return \"Not found\"\n",
    "    file_path = matches.values[0]\n",
    "\n",
    "    if isinstance(file_path, str) and len(file_path) > 0:\n",
    "        return file_path.split(\".\")[-1]\n",
    "    else:\n",
    "        return \"None\"\n",
    "\n",
    "\n",
    "result_df[\"attachment_type\"] = result_df[\"question\"].apply(find_attachment)\n",
    "\n",
    "\n",
    "def extract_tool_calls(code):\n",
    "    regex = r\"\\b(\\w+)\\(\"\n",
    "    function_calls = [el for el in re.findall(regex, code) if el.islower()]\n",
    "\n",
    "    function_call_counter = Counter(function_calls)\n",
    "    return function_call_counter\n",
    "\n",
    "\n",
    "def sum_tool_calls(steps):\n",
    "    total_count = Counter()\n",
    "    for step in steps:\n",
    "        if \"llm_output\" in step:\n",
    "            total_count += extract_tool_calls(step[\"llm_output\"])\n",
    "\n",
    "    return total_count\n",
    "\n",
    "\n",
    "def get_durations(row):\n",
    "    # start_datetime = datetime.strptime(row['start_time'], \"%Y-%m-%d %H:%M:%S\")\n",
    "    # end_datetime = datetime.strptime(row['end_time'], \"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "    duration_timedelta = row[\"end_time\"] - row[\"start_time\"]\n",
    "    return int(duration_timedelta.total_seconds())\n",
    "\n",
    "\n",
    "result_df[\"duration\"] = result_df.apply(get_durations, axis=1)\n",
    "# result_df[\"tool_calls\"] = result_df[\"intermediate_steps\"].apply(sum_tool_calls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df[\"agent_name\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Inspect specific runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_df = result_df\n",
    "# sel_df = sel_df.loc[\n",
    "#     (result_df[\"agent_name\"].isin(list_versions))\n",
    "# ]\n",
    "sel_df = sel_df.reset_index(drop=True)\n",
    "display(sel_df[\"agent_name\"].value_counts())\n",
    "sel_df = sel_df.drop_duplicates(subset=[\"agent_name\", \"question\"])\n",
    "display(sel_df.groupby(\"agent_name\")[[\"task\"]].value_counts())\n",
    "print(\"Total length:\", len(sel_df), \"- is complete:\", len(sel_df) == 165)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(\"Average score:\", sel_df.groupby(\"agent_name\")[[\"is_correct\"]].mean().round(3))\n",
    "display(\n",
    "    sel_df.groupby([\"agent_name\", \"task\"])[[\"is_correct\", \"is_near_correct\", \"count_steps\", \"question\", \"duration\"]]\n",
    "    .agg(\n",
    "        {\n",
    "            \"is_correct\": \"mean\",\n",
    "            \"is_near_correct\": \"mean\",\n",
    "            \"count_steps\": \"mean\",\n",
    "            \"question\": \"count\",\n",
    "            \"duration\": \"mean\",\n",
    "        }\n",
    "    )\n",
    "    .rename(columns={\"question\": \"count\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "\n",
    "cumulative_df = (\n",
    "    (\n",
    "        sel_df.groupby(\"agent_name\")[[\"is_correct\", \"is_near_correct\"]]\n",
    "        .expanding(min_periods=1, axis=0, method=\"single\")\n",
    "        .agg({\"is_correct\": \"mean\", \"is_near_correct\": \"count\"})\n",
    "        .reset_index()\n",
    "    )\n",
    "    .copy()\n",
    "    .rename(columns={\"is_near_correct\": \"index\"})\n",
    ")\n",
    "cumulative_df[\"index\"] = cumulative_df[\"index\"].astype(int) - 1\n",
    "\n",
    "\n",
    "def find_question(row):\n",
    "    try:\n",
    "        res = sel_df.loc[sel_df[\"agent_name\"] == row[\"agent_name\"], \"question\"].iloc[row[\"index\"]][:50]\n",
    "        return res\n",
    "    except Exception:\n",
    "        return \"\"\n",
    "\n",
    "\n",
    "cumulative_df[\"question\"] = cumulative_df.apply(find_question, axis=1)\n",
    "\n",
    "px.line(\n",
    "    cumulative_df,\n",
    "    color=\"agent_name\",\n",
    "    x=\"index\",\n",
    "    y=\"is_correct\",\n",
    "    hover_data=\"question\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Dive deeper into one run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_df = result_df.loc[result_df[\"agent_name\"] == \"o1\"]\n",
    "print(len(sel_df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "error_types = [\n",
    "    \"AgentParsingError\",\n",
    "    \"AgentExecutionError\",\n",
    "    \"AgentMaxIterationsError\",\n",
    "    \"AgentGenerationError\",\n",
    "]\n",
    "sel_df[error_types] = 0\n",
    "sel_df[\"Count steps\"] = np.nan\n",
    "\n",
    "\n",
    "def count_errors(row):\n",
    "    if isinstance(row[\"intermediate_steps\"], list):\n",
    "        row[\"Count steps\"] = len(row[\"intermediate_steps\"])\n",
    "        for step in row[\"intermediate_steps\"]:\n",
    "            if isinstance(step, dict) and \"error\" in step:\n",
    "                try:\n",
    "                    row[str(step[\"error\"][\"error_type\"])] += 1\n",
    "                except Exception:\n",
    "                    pass\n",
    "    return row\n",
    "\n",
    "\n",
    "sel_df = sel_df.apply(count_errors, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "\n",
    "aggregate_errors = (\n",
    "    sel_df.groupby([\"is_correct\"])[error_types + [\"Count steps\"]].mean().reset_index().melt(id_vars=[\"is_correct\"])\n",
    ")\n",
    "\n",
    "fig = px.bar(\n",
    "    aggregate_errors,\n",
    "    y=\"value\",\n",
    "    x=\"variable\",\n",
    "    color=\"is_correct\",\n",
    "    labels={\n",
    "        \"agent_name\": \"<b>Model</b>\",\n",
    "        \"task\": \"<b>Level</b>\",\n",
    "        \"aggregate_score\": \"<b>Performance</b>\",\n",
    "        \"value\": \"<b>Average count</b>\",\n",
    "        \"eval_score_GPT4\": \"<b>Score</b>\",\n",
    "    },\n",
    ")\n",
    "fig.update_layout(\n",
    "    height=500,\n",
    "    width=800,\n",
    "    barmode=\"group\",\n",
    "    bargroupgap=0.0,\n",
    ")\n",
    "fig.update_traces(textposition=\"outside\")\n",
    "fig.write_image(\"aggregate_errors.png\", scale=3)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect result by file extension type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(\n",
    "    result_df.groupby([\"attachment_type\"])[[\"is_correct\", \"count_steps\", \"question\"]].agg(\n",
    "        {\"is_correct\": \"mean\", \"count_steps\": \"mean\", \"question\": \"count\"}\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Ensembling methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = result_df[\"agent_name\"].value_counts()\n",
    "long_series = result_df.loc[result_df[\"agent_name\"].isin(counts[counts > 140].index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def majority_vote(df):\n",
    "    df = df[(df[\"prediction\"] != \"Unable to determine\") & (~df[\"prediction\"].isna()) & (df[\"prediction\"] != \"None\")]\n",
    "\n",
    "    answer_modes = df.groupby(\"question\")[\"prediction\"].agg(lambda x: x.mode()[0]).reset_index()\n",
    "    first_occurrences = (\n",
    "        df.groupby([\"question\", \"prediction\"]).agg({\"task\": \"first\", \"is_correct\": \"first\"}).reset_index()\n",
    "    )\n",
    "    result = answer_modes.merge(first_occurrences, on=[\"question\", \"prediction\"], how=\"left\")\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "def oracle(df):\n",
    "    def get_first_correct_or_first_wrong(group):\n",
    "        correct_answers = group[group[\"is_correct\"]]\n",
    "        if len(correct_answers) > 0:\n",
    "            return correct_answers.iloc[0]\n",
    "        return group.iloc[0]\n",
    "\n",
    "    result = df.groupby(\"question\").apply(get_first_correct_or_first_wrong)\n",
    "\n",
    "    return result.reset_index(drop=True)\n",
    "\n",
    "\n",
    "display((long_series.groupby(\"agent_name\")[\"is_correct\"].mean() * 100).round(2))\n",
    "print(f\"Majority score: {majority_vote(long_series)['is_correct'].mean() * 100:.2f}\")\n",
    "print(f\"Oracle score: {oracle(long_series)['is_correct'].mean() * 100:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_run = \"code_o1_04_february_submission5.jsonl\"\n",
    "df = pd.read_json(f\"output/validation/{agent_run}\", lines=True)\n",
    "df = df[[\"task_id\", \"prediction\", \"intermediate_steps\"]]\n",
    "df = df.rename(columns={\"prediction\": \"model_answer\", \"intermediate_steps\": \"reasoning_trace\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json(\"submission.jsonl\", orient=\"records\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agents",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
